{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input, inplace = False):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self, inplace = False):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.inplace = inplace\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input, inplace = self.inplace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input, inplace = False):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self, inplace = False):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.inplace = inplace\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input, inplace = self.inplace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicConv2d(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, activation = 'relu', **kwargs):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(input_channels, output_channels, bias=False, **kwargs)\n",
    "        self.bn = nn.BatchNorm2d(output_channels)\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            self.relu = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            self.relu = swish(inplace = True)\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            self.relu = mish(inplace = True)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        x = self.relu(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "#same naive inception module\n",
    "class InceptionA(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, pool_features, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.branch1x1 = BasicConv2d(input_channels, 64, kernel_size=1, activation = activation)\n",
    "\n",
    "        self.branch5x5 = nn.Sequential(\n",
    "            BasicConv2d(input_channels, 48, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(48, 64, kernel_size=5, padding=2, activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branch3x3 = nn.Sequential(\n",
    "            BasicConv2d(input_channels, 64, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(64, 96, kernel_size=3, padding=1, activation = activation),\n",
    "            BasicConv2d(96, 96, kernel_size=3, padding=1, activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branchpool = nn.Sequential(\n",
    "            nn.AvgPool2d(kernel_size=3, stride=1, padding=1),\n",
    "            BasicConv2d(input_channels, pool_features, kernel_size=3, padding=1, activation = activation)\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        #x -> 1x1(same)\n",
    "        branch1x1 = self.branch1x1(x)\n",
    "\n",
    "        #x -> 1x1 -> 5x5(same)\n",
    "        branch5x5 = self.branch5x5(x)\n",
    "        #branch5x5 = self.branch5x5_2(branch5x5)\n",
    "\n",
    "        #x -> 1x1 -> 3x3 -> 3x3(same)\n",
    "        branch3x3 = self.branch3x3(x)\n",
    "\n",
    "        #x -> pool -> 1x1(same)\n",
    "        branchpool = self.branchpool(x)\n",
    "\n",
    "        outputs = [branch1x1, branch5x5, branch3x3, branchpool]\n",
    "\n",
    "        return torch.cat(outputs, 1)\n",
    "\n",
    "#downsample\n",
    "#Factorization into smaller convolutions\n",
    "class InceptionB(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.branch3x3 = BasicConv2d(input_channels, 384, kernel_size=3, stride=2, activation = activation)\n",
    "\n",
    "        self.branch3x3stack = nn.Sequential(\n",
    "            BasicConv2d(input_channels, 64, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(64, 96, kernel_size=3, padding=1, activation = activation),\n",
    "            BasicConv2d(96, 96, kernel_size=3, stride=2, activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branchpool = nn.MaxPool2d(kernel_size=3, stride=2)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        #x - > 3x3(downsample)\n",
    "        branch3x3 = self.branch3x3(x)\n",
    "\n",
    "        #x -> 3x3 -> 3x3(downsample)\n",
    "        branch3x3stack = self.branch3x3stack(x)\n",
    "\n",
    "        #x -> avgpool(downsample)\n",
    "        branchpool = self.branchpool(x)\n",
    "\n",
    "        #\"\"\"We can use two parallel stride 2 blocks: P and C. P is a pooling \n",
    "        #layer (either average or maximum pooling) the activation, both of \n",
    "        #them are stride 2 the filter banks of which are concatenated as in \n",
    "        #figure 10.\"\"\"\n",
    "        outputs = [branch3x3, branch3x3stack, branchpool]\n",
    "\n",
    "        return torch.cat(outputs, 1)\n",
    "    \n",
    "#Factorizing Convolutions with Large Filter Size\n",
    "class InceptionC(nn.Module):\n",
    "    def __init__(self, input_channels, channels_7x7, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.branch1x1 = BasicConv2d(input_channels, 192, kernel_size=1, activation = activation)\n",
    "\n",
    "        c7 = channels_7x7\n",
    "\n",
    "        #In theory, we could go even further and argue that one can replace any n × n \n",
    "        #convolution by a 1 × n convolution followed by a n × 1 convolution and the \n",
    "        #computational cost saving increases dramatically as n grows (see figure 6).\n",
    "        self.branch7x7 = nn.Sequential(\n",
    "            BasicConv2d(input_channels, c7, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0), activation = activation),\n",
    "            BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3), activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branch7x7stack = nn.Sequential(\n",
    "            BasicConv2d(input_channels, c7, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0), activation = activation),\n",
    "            BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3), activation = activation),\n",
    "            BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0), activation = activation),\n",
    "            BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3), activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branch_pool = nn.Sequential(\n",
    "            nn.AvgPool2d(kernel_size=3, stride=1, padding=1),\n",
    "            BasicConv2d(input_channels, 192, kernel_size=1, activation = activation),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        #x -> 1x1(same)\n",
    "        branch1x1 = self.branch1x1(x)\n",
    "\n",
    "        #x -> 1layer 1*7 and 7*1 (same)\n",
    "        branch7x7 = self.branch7x7(x)\n",
    "\n",
    "        #x-> 2layer 1*7 and 7*1(same)\n",
    "        branch7x7stack = self.branch7x7stack(x)\n",
    "\n",
    "        #x-> avgpool (same)\n",
    "        branchpool = self.branch_pool(x)\n",
    "\n",
    "        outputs = [branch1x1, branch7x7, branch7x7stack, branchpool]\n",
    "\n",
    "        return torch.cat(outputs, 1)\n",
    "\n",
    "class InceptionD(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.branch3x3 = nn.Sequential(\n",
    "            BasicConv2d(input_channels, 192, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(192, 320, kernel_size=3, stride=2, activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branch7x7 = nn.Sequential(\n",
    "            BasicConv2d(input_channels, 192, kernel_size=1, activation = activation),\n",
    "            BasicConv2d(192, 192, kernel_size=(1, 7), padding=(0, 3), activation = activation),\n",
    "            BasicConv2d(192, 192, kernel_size=(7, 1), padding=(3, 0), activation = activation),\n",
    "            BasicConv2d(192, 192, kernel_size=3, stride=2, activation = activation)\n",
    "        )\n",
    "\n",
    "        self.branchpool = nn.AvgPool2d(kernel_size=3, stride=2)\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        #x -> 1x1 -> 3x3(downsample)\n",
    "        branch3x3 = self.branch3x3(x)\n",
    "\n",
    "        #x -> 1x1 -> 1x7 -> 7x1 -> 3x3 (downsample)\n",
    "        branch7x7 = self.branch7x7(x)\n",
    "\n",
    "        #x -> avgpool (downsample)\n",
    "        branchpool = self.branchpool(x)\n",
    "\n",
    "        outputs = [branch3x3, branch7x7, branchpool]\n",
    "\n",
    "        return torch.cat(outputs, 1)\n",
    "    \n",
    "\n",
    "#same\n",
    "class InceptionE(nn.Module):\n",
    "    def __init__(self, input_channels, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.branch1x1 = BasicConv2d(input_channels, 320, kernel_size=1, activation = activation)\n",
    "\n",
    "        self.branch3x3_1 = BasicConv2d(input_channels, 384, kernel_size=1, activation = activation)\n",
    "        self.branch3x3_2a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1), activation = activation)\n",
    "        self.branch3x3_2b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0), activation = activation)\n",
    "            \n",
    "        self.branch3x3stack_1 = BasicConv2d(input_channels, 448, kernel_size=1, activation = activation)\n",
    "        self.branch3x3stack_2 = BasicConv2d(448, 384, kernel_size=3, padding=1, activation = activation)\n",
    "        self.branch3x3stack_3a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1), activation = activation)\n",
    "        self.branch3x3stack_3b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0), activation = activation)\n",
    "\n",
    "        self.branch_pool = nn.Sequential(\n",
    "            nn.AvgPool2d(kernel_size=3, stride=1, padding=1),\n",
    "            BasicConv2d(input_channels, 192, kernel_size=1, activation = activation)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        #x -> 1x1 (same)\n",
    "        branch1x1 = self.branch1x1(x)\n",
    "\n",
    "        # x -> 1x1 -> 3x1\n",
    "        # x -> 1x1 -> 1x3\n",
    "        # concatenate(3x1, 1x3)\n",
    "        #\"\"\"7. Inception modules with expanded the filter bank outputs. \n",
    "        #This architecture is used on the coarsest (8 × 8) grids to promote \n",
    "        #high dimensional representations, as suggested by principle \n",
    "        #2 of Section 2.\"\"\"\n",
    "        branch3x3 = self.branch3x3_1(x)\n",
    "        branch3x3 = [\n",
    "            self.branch3x3_2a(branch3x3),\n",
    "            self.branch3x3_2b(branch3x3)\n",
    "        ]\n",
    "        branch3x3 = torch.cat(branch3x3, 1)\n",
    "\n",
    "        # x -> 1x1 -> 3x3 -> 1x3\n",
    "        # x -> 1x1 -> 3x3 -> 3x1\n",
    "        #concatenate(1x3, 3x1)\n",
    "        branch3x3stack = self.branch3x3stack_1(x)\n",
    "        branch3x3stack = self.branch3x3stack_2(branch3x3stack)\n",
    "        branch3x3stack = [\n",
    "            self.branch3x3stack_3a(branch3x3stack),\n",
    "            self.branch3x3stack_3b(branch3x3stack)\n",
    "        ]\n",
    "        branch3x3stack = torch.cat(branch3x3stack, 1)\n",
    "\n",
    "        branchpool = self.branch_pool(x)\n",
    "\n",
    "        outputs = [branch1x1, branch3x3, branch3x3stack, branchpool]\n",
    "\n",
    "        return torch.cat(outputs, 1)\n",
    "\n",
    "class InceptionV3(nn.Module):\n",
    "    \n",
    "    def __init__(self, num_classes=10, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.Conv2d_1a_3x3 = BasicConv2d(3, 32, kernel_size=3, padding=1, activation = activation)\n",
    "        self.Conv2d_2a_3x3 = BasicConv2d(32, 32, kernel_size=3, padding=1, activation = activation)\n",
    "        self.Conv2d_2b_3x3 = BasicConv2d(32, 64, kernel_size=3, padding=1, activation = activation)\n",
    "        self.Conv2d_3b_1x1 = BasicConv2d(64, 80, kernel_size=1, activation = activation)\n",
    "        self.Conv2d_4a_3x3 = BasicConv2d(80, 192, kernel_size=3, activation = activation)\n",
    "\n",
    "        #naive inception module\n",
    "        self.Mixed_5b = InceptionA(192, pool_features=32, activation = activation)\n",
    "        self.Mixed_5c = InceptionA(256, pool_features=64, activation = activation)\n",
    "        self.Mixed_5d = InceptionA(288, pool_features=64, activation = activation)\n",
    "\n",
    "        #downsample\n",
    "        self.Mixed_6a = InceptionB(288, activation = activation)\n",
    "\n",
    "        self.Mixed_6b = InceptionC(768, channels_7x7=128, activation = activation)\n",
    "        self.Mixed_6c = InceptionC(768, channels_7x7=160, activation = activation)\n",
    "        self.Mixed_6d = InceptionC(768, channels_7x7=160, activation = activation)\n",
    "        self.Mixed_6e = InceptionC(768, channels_7x7=192, activation = activation)\n",
    "\n",
    "        #downsample\n",
    "        self.Mixed_7a = InceptionD(768, activation = activation)\n",
    "\n",
    "        self.Mixed_7b = InceptionE(1280, activation = activation)\n",
    "        self.Mixed_7c = InceptionE(2048, activation = activation)\n",
    "        \n",
    "        #6*6 feature size\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.dropout = nn.Dropout2d()\n",
    "        self.linear = nn.Linear(2048, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        #32 -> 30\n",
    "        x = self.Conv2d_1a_3x3(x)\n",
    "        x = self.Conv2d_2a_3x3(x)\n",
    "        x = self.Conv2d_2b_3x3(x)\n",
    "        x = self.Conv2d_3b_1x1(x)\n",
    "        x = self.Conv2d_4a_3x3(x)\n",
    "\n",
    "        #30 -> 30\n",
    "        x = self.Mixed_5b(x)\n",
    "        x = self.Mixed_5c(x)\n",
    "        x = self.Mixed_5d(x)\n",
    "\n",
    "        #30 -> 14\n",
    "        #Efficient Grid Size Reduction to avoid representation\n",
    "        #bottleneck\n",
    "        x = self.Mixed_6a(x)\n",
    "\n",
    "        #14 -> 14\n",
    "        #\"\"\"In practice, we have found that employing this factorization does not \n",
    "        #work well on early layers, but it gives very good results on medium \n",
    "        #grid-sizes (On m × m feature maps, where m ranges between 12 and 20). \n",
    "        #On that level, very good results can be achieved by using 1 × 7 convolutions \n",
    "        #followed by 7 × 1 convolutions.\"\"\"\n",
    "        x = self.Mixed_6b(x)\n",
    "        x = self.Mixed_6c(x)\n",
    "        x = self.Mixed_6d(x)\n",
    "        x = self.Mixed_6e(x)\n",
    "\n",
    "        #14 -> 6\n",
    "        #Efficient Grid Size Reduction\n",
    "        x = self.Mixed_7a(x)\n",
    "\n",
    "        #6 -> 6\n",
    "        #We are using this solution only on the coarsest grid, \n",
    "        #since that is the place where producing high dimensional \n",
    "        #sparse representation is the most critical as the ratio of \n",
    "        #local processing (by 1 × 1 convolutions) is increased compared \n",
    "        #to the spatial aggregation.\"\"\"\n",
    "        x = self.Mixed_7b(x)\n",
    "        x = self.Mixed_7c(x)\n",
    "\n",
    "        #6 -> 1\n",
    "        x = self.avgpool(x)\n",
    "        x = self.dropout(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.linear(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "def inceptionv3(activation = 'relu'):\n",
    "    return InceptionV3(activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 28307736.60it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = inceptionv3(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 258.2529.. Average time per step: 0.6605.. Train loss: 1.6494.. Train accuracy: 0.3758.. Top-3 train accuracy: 0.7361.. Test loss: 1.6536.. Test accuracy: 0.4223.. Top-3 test accuracy: 0.7733\n",
      "Epoch 2/100.. Time per epoch: 256.9586.. Average time per step: 0.6572.. Train loss: 1.2841.. Train accuracy: 0.5294.. Top-3 train accuracy: 0.8424.. Test loss: 1.1048.. Test accuracy: 0.5961.. Top-3 test accuracy: 0.8795\n",
      "Epoch 3/100.. Time per epoch: 257.0979.. Average time per step: 0.6575.. Train loss: 1.1006.. Train accuracy: 0.6029.. Top-3 train accuracy: 0.8786.. Test loss: 0.9879.. Test accuracy: 0.6528.. Top-3 test accuracy: 0.8989\n",
      "Epoch 4/100.. Time per epoch: 257.2303.. Average time per step: 0.6579.. Train loss: 0.9694.. Train accuracy: 0.6528.. Top-3 train accuracy: 0.9037.. Test loss: 0.8478.. Test accuracy: 0.7012.. Top-3 test accuracy: 0.9224\n",
      "Epoch 5/100.. Time per epoch: 257.2799.. Average time per step: 0.6580.. Train loss: 0.8626.. Train accuracy: 0.6939.. Top-3 train accuracy: 0.9195.. Test loss: 0.9461.. Test accuracy: 0.6718.. Top-3 test accuracy: 0.9177\n",
      "Epoch 6/100.. Time per epoch: 257.0746.. Average time per step: 0.6575.. Train loss: 0.7809.. Train accuracy: 0.7252.. Top-3 train accuracy: 0.9325.. Test loss: 0.6554.. Test accuracy: 0.7729.. Top-3 test accuracy: 0.9479\n",
      "Epoch 7/100.. Time per epoch: 257.1363.. Average time per step: 0.6576.. Train loss: 0.7174.. Train accuracy: 0.7491.. Top-3 train accuracy: 0.9397.. Test loss: 0.6678.. Test accuracy: 0.7695.. Top-3 test accuracy: 0.9538\n",
      "Epoch 8/100.. Time per epoch: 257.2068.. Average time per step: 0.6578.. Train loss: 0.6617.. Train accuracy: 0.7714.. Top-3 train accuracy: 0.9474.. Test loss: 0.5962.. Test accuracy: 0.7962.. Top-3 test accuracy: 0.9578\n",
      "Epoch 9/100.. Time per epoch: 256.9704.. Average time per step: 0.6572.. Train loss: 0.6237.. Train accuracy: 0.7833.. Top-3 train accuracy: 0.9531.. Test loss: 0.5520.. Test accuracy: 0.8066.. Top-3 test accuracy: 0.9667\n",
      "Epoch 10/100.. Time per epoch: 256.7363.. Average time per step: 0.6566.. Train loss: 0.5856.. Train accuracy: 0.7967.. Top-3 train accuracy: 0.9564.. Test loss: 0.4937.. Test accuracy: 0.8321.. Top-3 test accuracy: 0.9697\n",
      "Epoch 11/100.. Time per epoch: 256.7557.. Average time per step: 0.6567.. Train loss: 0.5538.. Train accuracy: 0.8079.. Top-3 train accuracy: 0.9620.. Test loss: 0.5082.. Test accuracy: 0.8258.. Top-3 test accuracy: 0.9672\n",
      "Epoch 12/100.. Time per epoch: 256.8023.. Average time per step: 0.6568.. Train loss: 0.5266.. Train accuracy: 0.8183.. Top-3 train accuracy: 0.9639.. Test loss: 0.4518.. Test accuracy: 0.8459.. Top-3 test accuracy: 0.9723\n",
      "Epoch 13/100.. Time per epoch: 256.7136.. Average time per step: 0.6566.. Train loss: 0.4982.. Train accuracy: 0.8275.. Top-3 train accuracy: 0.9671.. Test loss: 0.4657.. Test accuracy: 0.8398.. Top-3 test accuracy: 0.9732\n",
      "Epoch 14/100.. Time per epoch: 256.8184.. Average time per step: 0.6568.. Train loss: 0.4741.. Train accuracy: 0.8349.. Top-3 train accuracy: 0.9703.. Test loss: 0.4436.. Test accuracy: 0.8453.. Top-3 test accuracy: 0.9762\n",
      "Epoch 15/100.. Time per epoch: 256.7946.. Average time per step: 0.6568.. Train loss: 0.4578.. Train accuracy: 0.8425.. Top-3 train accuracy: 0.9711.. Test loss: 0.4117.. Test accuracy: 0.8587.. Top-3 test accuracy: 0.9776\n",
      "Epoch 16/100.. Time per epoch: 256.5748.. Average time per step: 0.6562.. Train loss: 0.4355.. Train accuracy: 0.8496.. Top-3 train accuracy: 0.9731.. Test loss: 0.4386.. Test accuracy: 0.8488.. Top-3 test accuracy: 0.9759\n",
      "Epoch 17/100.. Time per epoch: 256.5257.. Average time per step: 0.6561.. Train loss: 0.4200.. Train accuracy: 0.8533.. Top-3 train accuracy: 0.9761.. Test loss: 0.4243.. Test accuracy: 0.8573.. Top-3 test accuracy: 0.9752\n",
      "Epoch 18/100.. Time per epoch: 256.6932.. Average time per step: 0.6565.. Train loss: 0.4022.. Train accuracy: 0.8598.. Top-3 train accuracy: 0.9774.. Test loss: 0.3834.. Test accuracy: 0.8640.. Top-3 test accuracy: 0.9788\n",
      "Epoch 19/100.. Time per epoch: 256.1931.. Average time per step: 0.6552.. Train loss: 0.3878.. Train accuracy: 0.8648.. Top-3 train accuracy: 0.9784.. Test loss: 0.3638.. Test accuracy: 0.8770.. Top-3 test accuracy: 0.9826\n",
      "Epoch 20/100.. Time per epoch: 256.3560.. Average time per step: 0.6556.. Train loss: 0.3666.. Train accuracy: 0.8727.. Top-3 train accuracy: 0.9803.. Test loss: 0.3642.. Test accuracy: 0.8794.. Top-3 test accuracy: 0.9800\n",
      "Epoch 21/100.. Time per epoch: 256.5912.. Average time per step: 0.6562.. Train loss: 0.3549.. Train accuracy: 0.8756.. Top-3 train accuracy: 0.9814.. Test loss: 0.3568.. Test accuracy: 0.8757.. Top-3 test accuracy: 0.9831\n",
      "Epoch 22/100.. Time per epoch: 256.4191.. Average time per step: 0.6558.. Train loss: 0.3433.. Train accuracy: 0.8792.. Top-3 train accuracy: 0.9830.. Test loss: 0.3309.. Test accuracy: 0.8852.. Top-3 test accuracy: 0.9836\n",
      "Epoch 23/100.. Time per epoch: 256.7159.. Average time per step: 0.6566.. Train loss: 0.3334.. Train accuracy: 0.8831.. Top-3 train accuracy: 0.9832.. Test loss: 0.3557.. Test accuracy: 0.8786.. Top-3 test accuracy: 0.9829\n",
      "Epoch 24/100.. Time per epoch: 256.5771.. Average time per step: 0.6562.. Train loss: 0.3160.. Train accuracy: 0.8892.. Top-3 train accuracy: 0.9843.. Test loss: 0.3416.. Test accuracy: 0.8876.. Top-3 test accuracy: 0.9847\n",
      "Epoch 25/100.. Time per epoch: 256.4233.. Average time per step: 0.6558.. Train loss: 0.3071.. Train accuracy: 0.8927.. Top-3 train accuracy: 0.9856.. Test loss: 0.3402.. Test accuracy: 0.8894.. Top-3 test accuracy: 0.9830\n",
      "Epoch 26/100.. Time per epoch: 256.3676.. Average time per step: 0.6557.. Train loss: 0.2890.. Train accuracy: 0.8994.. Top-3 train accuracy: 0.9862.. Test loss: 0.3413.. Test accuracy: 0.8906.. Top-3 test accuracy: 0.9846\n",
      "Epoch 27/100.. Time per epoch: 256.4962.. Average time per step: 0.6560.. Train loss: 0.2842.. Train accuracy: 0.8995.. Top-3 train accuracy: 0.9875.. Test loss: 0.3604.. Test accuracy: 0.8854.. Top-3 test accuracy: 0.9839\n",
      "Epoch 28/100.. Time per epoch: 256.3442.. Average time per step: 0.6556.. Train loss: 0.2759.. Train accuracy: 0.9032.. Top-3 train accuracy: 0.9872.. Test loss: 0.3172.. Test accuracy: 0.8961.. Top-3 test accuracy: 0.9855\n",
      "Epoch 29/100.. Time per epoch: 256.3924.. Average time per step: 0.6557.. Train loss: 0.2642.. Train accuracy: 0.9088.. Top-3 train accuracy: 0.9883.. Test loss: 0.3310.. Test accuracy: 0.8927.. Top-3 test accuracy: 0.9862\n",
      "Epoch 30/100.. Time per epoch: 256.4195.. Average time per step: 0.6558.. Train loss: 0.2584.. Train accuracy: 0.9085.. Top-3 train accuracy: 0.9890.. Test loss: 0.3353.. Test accuracy: 0.8928.. Top-3 test accuracy: 0.9840\n",
      "Epoch 31/100.. Time per epoch: 256.5696.. Average time per step: 0.6562.. Train loss: 0.2478.. Train accuracy: 0.9135.. Top-3 train accuracy: 0.9897.. Test loss: 0.3215.. Test accuracy: 0.8984.. Top-3 test accuracy: 0.9850\n",
      "Epoch 32/100.. Time per epoch: 256.7006.. Average time per step: 0.6565.. Train loss: 0.2363.. Train accuracy: 0.9155.. Top-3 train accuracy: 0.9907.. Test loss: 0.3349.. Test accuracy: 0.8917.. Top-3 test accuracy: 0.9853\n",
      "Epoch 33/100.. Time per epoch: 256.7962.. Average time per step: 0.6568.. Train loss: 0.2357.. Train accuracy: 0.9168.. Top-3 train accuracy: 0.9905.. Test loss: 0.3147.. Test accuracy: 0.9000.. Top-3 test accuracy: 0.9869\n",
      "Epoch 34/100.. Time per epoch: 256.9062.. Average time per step: 0.6570.. Train loss: 0.2230.. Train accuracy: 0.9215.. Top-3 train accuracy: 0.9917.. Test loss: 0.2948.. Test accuracy: 0.9093.. Top-3 test accuracy: 0.9875\n",
      "Epoch 35/100.. Time per epoch: 256.7264.. Average time per step: 0.6566.. Train loss: 0.2119.. Train accuracy: 0.9249.. Top-3 train accuracy: 0.9918.. Test loss: 0.3184.. Test accuracy: 0.8978.. Top-3 test accuracy: 0.9876\n",
      "Epoch 36/100.. Time per epoch: 257.2426.. Average time per step: 0.6579.. Train loss: 0.2061.. Train accuracy: 0.9270.. Top-3 train accuracy: 0.9932.. Test loss: 0.3190.. Test accuracy: 0.9051.. Top-3 test accuracy: 0.9864\n",
      "Epoch 37/100.. Time per epoch: 257.6957.. Average time per step: 0.6591.. Train loss: 0.1979.. Train accuracy: 0.9298.. Top-3 train accuracy: 0.9933.. Test loss: 0.3384.. Test accuracy: 0.8969.. Top-3 test accuracy: 0.9845\n",
      "Epoch 38/100.. Time per epoch: 257.3531.. Average time per step: 0.6582.. Train loss: 0.1947.. Train accuracy: 0.9315.. Top-3 train accuracy: 0.9939.. Test loss: 0.3067.. Test accuracy: 0.9056.. Top-3 test accuracy: 0.9870\n",
      "Epoch 39/100.. Time per epoch: 256.6802.. Average time per step: 0.6565.. Train loss: 0.1884.. Train accuracy: 0.9346.. Top-3 train accuracy: 0.9936.. Test loss: 0.3133.. Test accuracy: 0.9039.. Top-3 test accuracy: 0.9877\n",
      "Epoch 40/100.. Time per epoch: 256.7584.. Average time per step: 0.6567.. Train loss: 0.1784.. Train accuracy: 0.9366.. Top-3 train accuracy: 0.9944.. Test loss: 0.3147.. Test accuracy: 0.9030.. Top-3 test accuracy: 0.9872\n",
      "Epoch 41/100.. Time per epoch: 256.8897.. Average time per step: 0.6570.. Train loss: 0.1728.. Train accuracy: 0.9388.. Top-3 train accuracy: 0.9947.. Test loss: 0.3137.. Test accuracy: 0.9053.. Top-3 test accuracy: 0.9869\n",
      "Epoch 42/100.. Time per epoch: 256.6539.. Average time per step: 0.6564.. Train loss: 0.1701.. Train accuracy: 0.9393.. Top-3 train accuracy: 0.9950.. Test loss: 0.3315.. Test accuracy: 0.8971.. Top-3 test accuracy: 0.9876\n",
      "Epoch 43/100.. Time per epoch: 256.6616.. Average time per step: 0.6564.. Train loss: 0.1634.. Train accuracy: 0.9414.. Top-3 train accuracy: 0.9954.. Test loss: 0.3127.. Test accuracy: 0.9070.. Top-3 test accuracy: 0.9880\n",
      "Epoch 44/100.. Time per epoch: 256.7927.. Average time per step: 0.6568.. Train loss: 0.1584.. Train accuracy: 0.9451.. Top-3 train accuracy: 0.9954.. Test loss: 0.3309.. Test accuracy: 0.9021.. Top-3 test accuracy: 0.9855\n",
      "Epoch 45/100.. Time per epoch: 256.4165.. Average time per step: 0.6558.. Train loss: 0.1495.. Train accuracy: 0.9477.. Top-3 train accuracy: 0.9960.. Test loss: 0.3256.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9888\n",
      "Epoch 46/100.. Time per epoch: 256.4226.. Average time per step: 0.6558.. Train loss: 0.1512.. Train accuracy: 0.9462.. Top-3 train accuracy: 0.9958.. Test loss: 0.3378.. Test accuracy: 0.9028.. Top-3 test accuracy: 0.9871\n",
      "Epoch 47/100.. Time per epoch: 256.5264.. Average time per step: 0.6561.. Train loss: 0.1453.. Train accuracy: 0.9480.. Top-3 train accuracy: 0.9961.. Test loss: 0.3295.. Test accuracy: 0.9070.. Top-3 test accuracy: 0.9874\n",
      "Epoch 48/100.. Time per epoch: 256.4213.. Average time per step: 0.6558.. Train loss: 0.1401.. Train accuracy: 0.9504.. Top-3 train accuracy: 0.9965.. Test loss: 0.3495.. Test accuracy: 0.9003.. Top-3 test accuracy: 0.9859\n",
      "Epoch 49/100.. Time per epoch: 256.4703.. Average time per step: 0.6559.. Train loss: 0.1334.. Train accuracy: 0.9536.. Top-3 train accuracy: 0.9967.. Test loss: 0.3307.. Test accuracy: 0.9056.. Top-3 test accuracy: 0.9882\n",
      "Epoch 50/100.. Time per epoch: 256.2830.. Average time per step: 0.6555.. Train loss: 0.1297.. Train accuracy: 0.9537.. Top-3 train accuracy: 0.9965.. Test loss: 0.3094.. Test accuracy: 0.9132.. Top-3 test accuracy: 0.9888\n",
      "Epoch 51/100.. Time per epoch: 256.1015.. Average time per step: 0.6550.. Train loss: 0.1313.. Train accuracy: 0.9530.. Top-3 train accuracy: 0.9975.. Test loss: 0.3346.. Test accuracy: 0.9072.. Top-3 test accuracy: 0.9873\n",
      "Epoch 52/100.. Time per epoch: 255.9345.. Average time per step: 0.6546.. Train loss: 0.1273.. Train accuracy: 0.9552.. Top-3 train accuracy: 0.9969.. Test loss: 0.3230.. Test accuracy: 0.9070.. Top-3 test accuracy: 0.9869\n",
      "Epoch 53/100.. Time per epoch: 255.9621.. Average time per step: 0.6546.. Train loss: 0.1144.. Train accuracy: 0.9597.. Top-3 train accuracy: 0.9977.. Test loss: 0.3277.. Test accuracy: 0.9091.. Top-3 test accuracy: 0.9875\n",
      "Epoch 54/100.. Time per epoch: 256.2537.. Average time per step: 0.6554.. Train loss: 0.1140.. Train accuracy: 0.9591.. Top-3 train accuracy: 0.9977.. Test loss: 0.3296.. Test accuracy: 0.9052.. Top-3 test accuracy: 0.9870\n",
      "Epoch 55/100.. Time per epoch: 256.0636.. Average time per step: 0.6549.. Train loss: 0.1153.. Train accuracy: 0.9591.. Top-3 train accuracy: 0.9977.. Test loss: 0.3085.. Test accuracy: 0.9136.. Top-3 test accuracy: 0.9882\n",
      "Epoch 56/100.. Time per epoch: 255.8737.. Average time per step: 0.6544.. Train loss: 0.1131.. Train accuracy: 0.9601.. Top-3 train accuracy: 0.9979.. Test loss: 0.3146.. Test accuracy: 0.9115.. Top-3 test accuracy: 0.9899\n",
      "Epoch 57/100.. Time per epoch: 255.7321.. Average time per step: 0.6540.. Train loss: 0.1050.. Train accuracy: 0.9634.. Top-3 train accuracy: 0.9977.. Test loss: 0.3330.. Test accuracy: 0.9102.. Top-3 test accuracy: 0.9875\n",
      "Epoch 58/100.. Time per epoch: 255.6993.. Average time per step: 0.6540.. Train loss: 0.1019.. Train accuracy: 0.9639.. Top-3 train accuracy: 0.9980.. Test loss: 0.3338.. Test accuracy: 0.9103.. Top-3 test accuracy: 0.9879\n",
      "Epoch 59/100.. Time per epoch: 255.6063.. Average time per step: 0.6537.. Train loss: 0.1047.. Train accuracy: 0.9637.. Top-3 train accuracy: 0.9982.. Test loss: 0.3356.. Test accuracy: 0.9089.. Top-3 test accuracy: 0.9878\n",
      "Epoch 60/100.. Time per epoch: 255.3601.. Average time per step: 0.6531.. Train loss: 0.0975.. Train accuracy: 0.9658.. Top-3 train accuracy: 0.9982.. Test loss: 0.3320.. Test accuracy: 0.9117.. Top-3 test accuracy: 0.9882\n",
      "Epoch 61/100.. Time per epoch: 255.7699.. Average time per step: 0.6541.. Train loss: 0.0955.. Train accuracy: 0.9656.. Top-3 train accuracy: 0.9982.. Test loss: 0.3204.. Test accuracy: 0.9124.. Top-3 test accuracy: 0.9877\n",
      "Epoch 62/100.. Time per epoch: 255.7074.. Average time per step: 0.6540.. Train loss: 0.0951.. Train accuracy: 0.9659.. Top-3 train accuracy: 0.9983.. Test loss: 0.3462.. Test accuracy: 0.9108.. Top-3 test accuracy: 0.9878\n",
      "Epoch 63/100.. Time per epoch: 255.7923.. Average time per step: 0.6542.. Train loss: 0.0967.. Train accuracy: 0.9665.. Top-3 train accuracy: 0.9981.. Test loss: 0.3334.. Test accuracy: 0.9104.. Top-3 test accuracy: 0.9884\n",
      "Epoch 64/100.. Time per epoch: 255.5583.. Average time per step: 0.6536.. Train loss: 0.0881.. Train accuracy: 0.9688.. Top-3 train accuracy: 0.9985.. Test loss: 0.3351.. Test accuracy: 0.9140.. Top-3 test accuracy: 0.9879\n",
      "Epoch 65/100.. Time per epoch: 255.4686.. Average time per step: 0.6534.. Train loss: 0.0882.. Train accuracy: 0.9686.. Top-3 train accuracy: 0.9985.. Test loss: 0.3503.. Test accuracy: 0.9098.. Top-3 test accuracy: 0.9882\n",
      "Epoch 66/100.. Time per epoch: 255.4293.. Average time per step: 0.6533.. Train loss: 0.0826.. Train accuracy: 0.9703.. Top-3 train accuracy: 0.9986.. Test loss: 0.3503.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9884\n",
      "Epoch 67/100.. Time per epoch: 255.3450.. Average time per step: 0.6531.. Train loss: 0.0842.. Train accuracy: 0.9705.. Top-3 train accuracy: 0.9987.. Test loss: 0.3426.. Test accuracy: 0.9140.. Top-3 test accuracy: 0.9892\n",
      "Epoch 68/100.. Time per epoch: 255.3080.. Average time per step: 0.6530.. Train loss: 0.0788.. Train accuracy: 0.9721.. Top-3 train accuracy: 0.9989.. Test loss: 0.4094.. Test accuracy: 0.9025.. Top-3 test accuracy: 0.9860\n",
      "Epoch 69/100.. Time per epoch: 255.4167.. Average time per step: 0.6532.. Train loss: 0.0808.. Train accuracy: 0.9714.. Top-3 train accuracy: 0.9987.. Test loss: 0.3431.. Test accuracy: 0.9112.. Top-3 test accuracy: 0.9887\n",
      "Epoch 70/100.. Time per epoch: 255.4331.. Average time per step: 0.6533.. Train loss: 0.0792.. Train accuracy: 0.9726.. Top-3 train accuracy: 0.9990.. Test loss: 0.3273.. Test accuracy: 0.9151.. Top-3 test accuracy: 0.9898\n",
      "Epoch 71/100.. Time per epoch: 255.4363.. Average time per step: 0.6533.. Train loss: 0.0759.. Train accuracy: 0.9731.. Top-3 train accuracy: 0.9989.. Test loss: 0.3694.. Test accuracy: 0.9100.. Top-3 test accuracy: 0.9872\n",
      "Epoch 72/100.. Time per epoch: 255.3868.. Average time per step: 0.6532.. Train loss: 0.0749.. Train accuracy: 0.9734.. Top-3 train accuracy: 0.9988.. Test loss: 0.3557.. Test accuracy: 0.9135.. Top-3 test accuracy: 0.9868\n",
      "Epoch 73/100.. Time per epoch: 255.4604.. Average time per step: 0.6534.. Train loss: 0.0719.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9989.. Test loss: 0.3767.. Test accuracy: 0.9084.. Top-3 test accuracy: 0.9886\n",
      "Epoch 74/100.. Time per epoch: 255.3858.. Average time per step: 0.6532.. Train loss: 0.0747.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9990.. Test loss: 0.3540.. Test accuracy: 0.9146.. Top-3 test accuracy: 0.9889\n",
      "Epoch 75/100.. Time per epoch: 255.3675.. Average time per step: 0.6531.. Train loss: 0.0677.. Train accuracy: 0.9760.. Top-3 train accuracy: 0.9991.. Test loss: 0.3477.. Test accuracy: 0.9138.. Top-3 test accuracy: 0.9881\n",
      "Epoch 76/100.. Time per epoch: 255.2130.. Average time per step: 0.6527.. Train loss: 0.0704.. Train accuracy: 0.9750.. Top-3 train accuracy: 0.9989.. Test loss: 0.3538.. Test accuracy: 0.9158.. Top-3 test accuracy: 0.9866\n",
      "Epoch 77/100.. Time per epoch: 255.0109.. Average time per step: 0.6522.. Train loss: 0.0688.. Train accuracy: 0.9762.. Top-3 train accuracy: 0.9992.. Test loss: 0.3493.. Test accuracy: 0.9150.. Top-3 test accuracy: 0.9884\n",
      "Epoch 78/100.. Time per epoch: 255.2826.. Average time per step: 0.6529.. Train loss: 0.0648.. Train accuracy: 0.9767.. Top-3 train accuracy: 0.9993.. Test loss: 0.3796.. Test accuracy: 0.9122.. Top-3 test accuracy: 0.9868\n",
      "Epoch 79/100.. Time per epoch: 255.0449.. Average time per step: 0.6523.. Train loss: 0.0645.. Train accuracy: 0.9776.. Top-3 train accuracy: 0.9993.. Test loss: 0.3459.. Test accuracy: 0.9161.. Top-3 test accuracy: 0.9885\n",
      "Epoch 80/100.. Time per epoch: 255.1561.. Average time per step: 0.6526.. Train loss: 0.0618.. Train accuracy: 0.9780.. Top-3 train accuracy: 0.9995.. Test loss: 0.3732.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9872\n",
      "Epoch 81/100.. Time per epoch: 255.4095.. Average time per step: 0.6532.. Train loss: 0.0627.. Train accuracy: 0.9781.. Top-3 train accuracy: 0.9991.. Test loss: 0.4003.. Test accuracy: 0.9079.. Top-3 test accuracy: 0.9875\n",
      "Epoch 82/100.. Time per epoch: 255.3106.. Average time per step: 0.6530.. Train loss: 0.0624.. Train accuracy: 0.9781.. Top-3 train accuracy: 0.9993.. Test loss: 0.3686.. Test accuracy: 0.9107.. Top-3 test accuracy: 0.9882\n",
      "Epoch 83/100.. Time per epoch: 255.2650.. Average time per step: 0.6529.. Train loss: 0.0566.. Train accuracy: 0.9799.. Top-3 train accuracy: 0.9993.. Test loss: 0.3931.. Test accuracy: 0.9108.. Top-3 test accuracy: 0.9885\n",
      "Epoch 84/100.. Time per epoch: 255.2887.. Average time per step: 0.6529.. Train loss: 0.0604.. Train accuracy: 0.9787.. Top-3 train accuracy: 0.9993.. Test loss: 0.3714.. Test accuracy: 0.9136.. Top-3 test accuracy: 0.9879\n",
      "Epoch 85/100.. Time per epoch: 255.1610.. Average time per step: 0.6526.. Train loss: 0.0590.. Train accuracy: 0.9793.. Top-3 train accuracy: 0.9993.. Test loss: 0.3542.. Test accuracy: 0.9162.. Top-3 test accuracy: 0.9888\n",
      "Epoch 86/100.. Time per epoch: 255.1232.. Average time per step: 0.6525.. Train loss: 0.0577.. Train accuracy: 0.9803.. Top-3 train accuracy: 0.9993.. Test loss: 0.3906.. Test accuracy: 0.9127.. Top-3 test accuracy: 0.9881\n",
      "Epoch 87/100.. Time per epoch: 255.1989.. Average time per step: 0.6527.. Train loss: 0.0554.. Train accuracy: 0.9814.. Top-3 train accuracy: 0.9995.. Test loss: 0.3771.. Test accuracy: 0.9184.. Top-3 test accuracy: 0.9877\n",
      "Epoch 88/100.. Time per epoch: 255.5073.. Average time per step: 0.6535.. Train loss: 0.0552.. Train accuracy: 0.9807.. Top-3 train accuracy: 0.9996.. Test loss: 0.3957.. Test accuracy: 0.9129.. Top-3 test accuracy: 0.9889\n",
      "Epoch 89/100.. Time per epoch: 255.8997.. Average time per step: 0.6545.. Train loss: 0.0555.. Train accuracy: 0.9810.. Top-3 train accuracy: 0.9993.. Test loss: 0.3823.. Test accuracy: 0.9127.. Top-3 test accuracy: 0.9899\n",
      "Epoch 90/100.. Time per epoch: 256.6684.. Average time per step: 0.6564.. Train loss: 0.0555.. Train accuracy: 0.9810.. Top-3 train accuracy: 0.9995.. Test loss: 0.3658.. Test accuracy: 0.9149.. Top-3 test accuracy: 0.9886\n",
      "Epoch 91/100.. Time per epoch: 256.6929.. Average time per step: 0.6565.. Train loss: 0.0520.. Train accuracy: 0.9820.. Top-3 train accuracy: 0.9994.. Test loss: 0.3550.. Test accuracy: 0.9164.. Top-3 test accuracy: 0.9890\n",
      "Epoch 92/100.. Time per epoch: 256.5134.. Average time per step: 0.6560.. Train loss: 0.0527.. Train accuracy: 0.9815.. Top-3 train accuracy: 0.9995.. Test loss: 0.3948.. Test accuracy: 0.9112.. Top-3 test accuracy: 0.9869\n",
      "Epoch 93/100.. Time per epoch: 256.7099.. Average time per step: 0.6565.. Train loss: 0.0513.. Train accuracy: 0.9821.. Top-3 train accuracy: 0.9994.. Test loss: 0.3842.. Test accuracy: 0.9137.. Top-3 test accuracy: 0.9867\n",
      "Epoch 94/100.. Time per epoch: 256.8496.. Average time per step: 0.6569.. Train loss: 0.0488.. Train accuracy: 0.9827.. Top-3 train accuracy: 0.9996.. Test loss: 0.3730.. Test accuracy: 0.9155.. Top-3 test accuracy: 0.9881\n",
      "Epoch 95/100.. Time per epoch: 256.9421.. Average time per step: 0.6571.. Train loss: 0.0508.. Train accuracy: 0.9819.. Top-3 train accuracy: 0.9992.. Test loss: 0.3526.. Test accuracy: 0.9196.. Top-3 test accuracy: 0.9899\n",
      "Epoch 96/100.. Time per epoch: 257.0664.. Average time per step: 0.6575.. Train loss: 0.0486.. Train accuracy: 0.9830.. Top-3 train accuracy: 0.9993.. Test loss: 0.4215.. Test accuracy: 0.9075.. Top-3 test accuracy: 0.9861\n",
      "Epoch 97/100.. Time per epoch: 255.9748.. Average time per step: 0.6547.. Train loss: 0.0461.. Train accuracy: 0.9837.. Top-3 train accuracy: 0.9997.. Test loss: 0.3983.. Test accuracy: 0.9121.. Top-3 test accuracy: 0.9885\n",
      "Epoch 98/100.. Time per epoch: 255.7241.. Average time per step: 0.6540.. Train loss: 0.0451.. Train accuracy: 0.9845.. Top-3 train accuracy: 0.9996.. Test loss: 0.4272.. Test accuracy: 0.9072.. Top-3 test accuracy: 0.9866\n",
      "Epoch 99/100.. Time per epoch: 255.6359.. Average time per step: 0.6538.. Train loss: 0.0500.. Train accuracy: 0.9829.. Top-3 train accuracy: 0.9994.. Test loss: 0.3731.. Test accuracy: 0.9166.. Top-3 test accuracy: 0.9893\n",
      "Epoch 100/100.. Time per epoch: 255.4680.. Average time per step: 0.6534.. Train loss: 0.0421.. Train accuracy: 0.9852.. Top-3 train accuracy: 0.9996.. Test loss: 0.4143.. Test accuracy: 0.9120.. Top-3 test accuracy: 0.9886\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_InceptionV3_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
